﻿Imports Microsoft.VisualBasic

Namespace SimilText

    Public Class Dictionary
        'Inherits Dictionary(Of String, UInteger)

        'Const BlochWords As Integer = 100000

        Public Enum FormatType
            TEXT = 1
            BIN = 2
        End Enum

        Shared Dictionary As Object

        Public Property Words As New Dictionary(Of String, UInteger)
        Public Property DicPathFile As String
        Public Property DicFormat As Integer
        Public Property DicBase As Dictionary

        'OTTIMIZZAZIONE per avere tutto in unit32
        Const OneUint As UInt32 = 1
        Private WordCount As UInt32
        Private WordBaseCount As UInt32

        Public _WordsList() As DictionaryElement
        Public Function WordsList(i As UInt32) As DictionaryElement
            If i > WordBaseCount Then
                Return Me._WordsList(i - WordBaseCount - OneUint)
            Else
                Return Me.DicBase._WordsList(i - OneUint)
            End If
        End Function

        'Public Property WordsList(i As UInt32) As DictionaryElement
        '    Get
        '        If i > Me.DicBase.Words.Count Then
        '            Return Me._WordsList(i - Me.DicBase.Words.Count - 1)
        '        Else
        '            Return Me.DicBase._WordsList(i - 1)
        '        End If
        '    End Get
        '    Set(value As DictionaryElement)
        '        Me._WordsList(i) = value
        '    End Set
        'End Property

        Public Shared Function GetStemming(PathFile As String) As Dictionary
            Dim c = HttpContext.Current.Cache
            Dim Dic = Dictionary
            Dim Key = "dic_stemming"

            Dic = c(Key)

            If Dic Is Nothing Then

                Utils.GC.WaitFinalizers()

                Dic = New Dictionary
                Dic.LoadDictionary(SF(PathFile, "it"), SimilText.Dictionary.FormatType.TEXT)

                c.Insert(Key, Dic)

            End If

            Return Dic


        End Function
        'Public Shared Function GetStemming() As Dictionary
        '    Dim c = HttpContext.Current.Cache
        '    Dim Dic = Dictionary
        '    Dim Key = "dic_stemming"

        '    Return c(Key)


        'End Function
        Public Shared Function GetStopWord(PathFile As String) As Dictionary
            Dim c = HttpContext.Current.Cache
            Dim Dic = Dictionary
            Dim Key = "dic_stopword"

            Dic = c(Key)

            If Dic Is Nothing Then

                Utils.GC.WaitFinalizers()

                Dic = New Dictionary
                Dic.LoadDictionary(SF(PathFile, "it"), SimilText.Dictionary.FormatType.TEXT)
                Dic.LoadDictionary(SF(PathFile, "en"), SimilText.Dictionary.FormatType.TEXT)

                c.Insert(Key, Dic)

            End If

            Return Dic

        End Function


        Public Sub New()



        End Sub

        Public Sub LoadDictionary(PathFile As String, f As FormatType)

            If Me.DicBase Is Nothing Then
                Me.DicBase = New Dictionary
            End If



            Me.DicPathFile = PathFile

            If _WordsList Is Nothing Then
                ReDim _WordsList(0)
            End If

            'wbaselist = Me.DicBase._WordsList

            'If Me.DicBase IsNot Nothing Then
            '    'Me.Words = Me.Words.Concat(Me.DicBase.Words.ToList)

            '    For Each w In Me.DicBase.Words
            '        Me.Words.Add(w.Key, w.Value)
            '    Next

            '    ReDim Me.WordsList(Me.DicBase.WordsList.Length - 1)
            '    Me.DicBase.WordsList.CopyTo(Me.WordsList, 0)
            'End If

            Select Case f
                Case FormatType.BIN
                    LoadDictionaryBin(PathFile)
                Case FormatType.TEXT
                    LoadDictionaryText(PathFile)
            End Select



        End Sub


        'Dim wbaselist() As DictionaryElement
        Public Sub LoadDictionaryBin(Path As String)
            Dim di() As Byte
            Dim LenFile As Integer

            Dim fri As New IO.BinaryReader(New IO.FileStream(Path, IO.FileMode.Open, IO.FileAccess.Read, IO.FileShare.Read))
            LenFile = fri.BaseStream.Length
            ReDim di(LenFile)
            fri.Read(di, 0, LenFile)
            fri.Close()

            'Words(0) = Nothing
            'WordsList = New List(Of DictionaryElement) From {New DictionaryElement}


            If LenFile > 0 Then
                Dim i = 0
                Dim TotWord = BitConverter.ToInt32(di, i)
                i += 4

                'ReDim WordsList(TotWord)

                For ct = 1 To TotWord
                    Dim w = New DictionaryElement
                    w.Load(di, i)
                    Me.AddWord(w)
                    'Me.WordsList.Add(w)
                    'Me.Words.Add(Me.WordsList(ct).Word, ct)
                Next

            End If


        End Sub

        Public Sub LoadDictionaryText(Path As String)

            'ReDim Me.WordsList(300000)

            Dim fsw = IO.File.OpenText(Path)

            Do Until fsw.EndOfStream
                Dim words = fsw.ReadLine.Split(",")
                Dim w = words(0)
                Dim Wordid As Integer

                Wordid = Me.AddWord(w)

                Dim we = Me.WordsList(Wordid)
                ReDim we.StemmingIds(words.Length - 2)

                For ct = 0 To words.Length - 2
                    w = words(ct + 1)

                    Wordid = Me.AddWord(w)

                    we.StemmingIds(ct) = Wordid

                Next

            Loop

            fsw.Close()
        End Sub

        'Public Sub LoadStopWord()

        '    Me.StopWords = New Dictionary(Of String, Integer)
        '    'Exit Sub

        '    Me.LoadStopWord(SF(StopWordsFile, "it"))
        '    Me.LoadStopWord(SF(StopWordsFile, "en"))

        'End Sub

        'Public Sub LoadStopWord(StopWordFile As String)

        '    'Me.StopWords = New Dictionary(Of String, Integer)
        '    'Exit Sub

        '    Dim fsw = IO.File.OpenText(StopWordFile)

        '    Do Until fsw.EndOfStream
        '        Dim sw = fsw.ReadLine()

        '        If Not Me.StopWords.ContainsKey(sw) Then
        '            Me.StopWords.Add(sw, 1)

        '        End If

        '    Loop

        '    fsw.Close()

        'End Sub

        'Public Sub LoadStemming()

        '    Me.LoadStemming(SF(StemmingFile, "it"))
        '    'Me.LoadStopWord(StemmingFile & "_en.txt")

        'End Sub
        'Public Sub LoadStemming(StemmingFile As String)

        '    Dim fsw = IO.File.OpenText(StemmingFile)

        '    Do Until fsw.EndOfStream
        '        Dim words = fsw.ReadLine.Split(",")
        '        Dim w = words(0)
        '        Dim Wordid As Integer

        '        Wordid = Me.AddWord(w)

        '        Dim we = Me.WordsList(Wordid)
        '        ReDim we.StemmingIds(words.Length - 2)

        '        For ct = 0 To words.Length - 2
        '            w = words(ct + 1)

        '            Wordid = Me.AddWord(w)

        '            we.StemmingIds(ct) = Wordid

        '        Next

        '    Loop

        '    fsw.Close()
        'End Sub

        Public Sub SaveDictionary()

            Dim fri As New IO.BinaryWriter(New IO.FileStream(DicPathFile, IO.FileMode.Open, IO.FileAccess.Write, IO.FileShare.Read))

            'Dim FirstWord = 0
            'If Me.DicBase IsNot Nothing Then
            '    FirstWord = Me.DicBase.Words.Count
            'End If

            'fri.Write(Me.Words.Count - FirstWord)
            fri.Write(Me.Words.Count)

            For ct = 1 To Me.Words.Count
                Me.WordsList(ct + Me.DicBase.Words.Count).Save(fri)
            Next

            fri.Close()

        End Sub
        Public Function AddWord(word As String) As Integer
            Dim WordDerID As Integer = -1
            Dim WordID As Integer



            If Me.Words.ContainsKey(word) Then
                WordID = Me.Words(word)
            ElseIf Me.DicBase.Words.ContainsKey(word) Then
                WordID = Me.DicBase.Words(word)
            Else
                Dim w = New DictionaryElement With {.Word = word}
                'verifica se è un hashtag
                If word.Substring(0, 1) = "#" And word.Length > 1 Then
                    w.IsHashtag = True

                    ReDim w.StemmingIds(0)
                    WordDerID = Me.AddWord(word.Substring(1))
                    w.StemmingIds(0) = WordDerID

                End If

                WordID = Me.AddWord(w)

                'If WordDerID <> -1 Then
                '    Dim wd = Me.WordsList(WordDerID)
                '    ReDim Preserve wd.DerivateIds(wd.DerivateIds.Length)
                '    wd.DerivateIds(wd.DerivateIds.Length - 1) = WordID
                'End If

            End If

            Return WordID

        End Function
        'Public Function AddWord2(word As String) As Integer
        '    Dim WordID As Integer
        '    Dim WordID_ht As Integer
        '    Dim IsHashtag = False

        '    If Me.Words.ContainsKey(word) Then
        '        WordID = Me.Words(word)
        '    Else


        '        If word.Substring(0, 1) = "#" And word.Length > 1 Then
        '            word = word.Trim("#")
        '            IsHashtag = True
        '        End If

        '        Dim w = New DictionaryElement With {.Word = word}
        '        Dim w_ht = New DictionaryElement With {.Word = "#" & word}

        '        WordID = Me.AddWord(w)
        '        WordID_ht = Me.AddWord(w_ht)

        '        ReDim w.DerivateIds(0)
        '        ReDim w_ht.DerivateIds(0)

        '        w.DerivateIds(0) = WordID_ht
        '        w_ht.DerivateIds(0) = WordID
        '        w_ht.IsHashtag = True

        '        If IsHashtag Then
        '            WordID = WordID_ht
        '        End If


        '    End If

        '    Return WordID

        'End Function


        Public Function AddWord(w As DictionaryElement) As Integer

            'Dim LastPos = Me.WordsList.Count
            'Me.WordsList.Add(w)

            Dim LastPos = Me.Words.Count '+ 1

            If LastPos >= _WordsList.Length Then
                ReDim Preserve _WordsList(_WordsList.Length + 100000)
            End If


            Me._WordsList(LastPos) = w

            LastPos += Me.DicBase.Words.Count + 1

            Me.Words.Add(w.Word, LastPos)

            'OTTIMIZZAZIONE per avere tutto in unit32
            WordCount = Me.Words.Count
            WordBaseCount = Me.DicBase.Words.Count


            Return LastPos

        End Function


        'Public Sub LoadDictionaryDB()
        '    Dim q = DB.GetDBQuery
        '    'Exit Sub

        '    Dim wId As Integer
        '    Dim Tempids() As String
        '    Dim tempid As UInteger

        '    'Dim maxid = (From der In DB.derivatedictionaries Select der.WordId).Max
        '    Dim maxid = DB.TabDictionary.SetSelect("Max(word_id) as max").ToObj(Of Integer)()

        '    ReDim Me.Words(maxid)
        '    For ct As Integer = 0 To Me.Words.Length - 1
        '        Me.Words(ct) = New DictionaryElement
        '    Next

        '    Dim q_der = DB.TabDictionary.NotifyChanged(False).ToList

        '    For Each d In q_der
        '        wId = d.Word_Id
        '        Me(d.Word) = wId
        '        Me.Words(wId).Word = d.Word
        '        If d.Word_Ref_Ids <> "" Then
        '            Tempids = Split(d.Word_Ref_Ids, ";")
        '            ReDim Me.Words(wId).DerivateIds(Tempids.Length - 1)
        '            For ct_id As Integer = 0 To Tempids.Length - 1
        '                If UInteger.TryParse(Tempids(ct_id), tempid) Then
        '                    Me.Words(wId).DerivateIds(ct_id) = tempid
        '                End If
        '                'Me.Words(WordId).DerivateIds(ct_id) = DbUtils.DBToLong(Tempids(ct_id))
        '            Next
        '        End If

        '    Next


        'End Sub

        'Public Function GetDerivateWords(ByVal word As String) As String()
        '    Dim s() As String = New String() {}

        '    Dim d As DictionaryElement = Me.Words(Me(word))

        '    If d.DerivateIds.Length > 0 Then
        '        ReDim s(d.DerivateIds.Length - 1)
        '        For ct As Integer = 0 To d.DerivateIds.Length - 1
        '            s(ct) = Me.Words(d.DerivateIds(ct)).Word
        '        Next
        '    End If

        '    Return s

        'End Function

    End Class
    'Public Class DictionaryGender

    '    Public Property MaleWords As Dictionary(Of String, Integer)
    '    Public Property FemaleWords As Dictionary(Of String, Integer)


    '    Public Sub New()

    '        Me.MaleWords = New Dictionary(Of String, Integer) From {
    '            {"around", 42},
    '            {"what", 35},
    '            {"more", 34},
    '            {"are", 28},
    '            {"as", 23},
    '            {"who", 19},
    '            {"below", 8},
    '            {"is", 8},
    '            {"these", 8},
    '            {"the", 7},
    '            {"a", 6},
    '            {"at", 6},
    '            {"it", 6},
    '            {"many", 6},
    '            {"said", 5},
    '            {"above", 4},
    '            {"to", 2}
    '        }



    '        Me.FemaleWords = New Dictionary(Of String, Integer) From {
    '            {"with", 52},
    '            {"if", 47},
    '            {"not", 27},
    '            {"where", 18},
    '            {"be", 17},
    '            {"when", 17},
    '            {"your", 17},
    '            {"her", 9},
    '            {"we", 8},
    '            {"should", 7},
    '            {"she", 6},
    '            {"and", 4},
    '            {"me", 4},
    '            {"myself", 4},
    '            {"hers", 3},
    '            {"was", 1}
    '        }


    '    End Sub

    'End Class


    Public Class DictionaryElement

        'Public WordId As Integer
        Public Property Word As String
        Public Property IsHashtag As Boolean = False
        Public Property StemmingIds As UInteger() = New UInteger() {}

        Public Sub Load(ByRef d() As Byte, ByRef Start As Integer)


            Dim DataLength As UInteger

            'leggo la lunghezza della parola
            DataLength = BitConverter.ToUInt32(d, Start)
            Start += 4

            'leggo la parola
            Dim DataText(DataLength - 1) As Byte
            Buffer.BlockCopy(d, Start, DataText, 0, DataLength)
            Me.Word = Encoding.UTF8.GetString(DataText)

            Start += DataLength

            ' leggo la lunghezza dei ref
            DataLength = BitConverter.ToUInt32(d, Start)
            Start += 4
            ' leggo i ref

            ReDim StemmingIds(DataLength - 1)

            Buffer.BlockCopy(d, Start, StemmingIds, 0, DataLength * 4)
            Start += DataLength * 4

            'hastag
            If Trim(Me.Word) <> "" Then

                If Me.Word.Substring(0, 1) = "#" Then
                    Me.IsHashtag = True
                End If

            End If


        End Sub

        Public Sub Save(ByRef fw As IO.BinaryWriter)

            Dim d = Encoding.UTF8.GetBytes(Me.Word)

            fw.Write(CType(d.Length, UInteger))

            fw.Write(d)

            ' scrivo lung ref
            fw.Write(CType(Me.StemmingIds.Length, UInteger))

            ' scrivo i dati ref
            Dim DataByte((Me.StemmingIds.Length * 4) - 1) As Byte
            Buffer.BlockCopy(Me.StemmingIds, 0, DataByte, 0, Me.StemmingIds.Length * 4)
            fw.Write(DataByte, 0, DataByte.Length)


        End Sub

    End Class


    'Public Class DerivatelDictionary
    '    Inherits Dictionary(Of String, DerivatelDictionaryElement)

    '    Public oDB As DBAccess

    '    Public Sub New(ByVal db As DBAccess)
    '        Me.oDB = db
    '    End Sub

    '    Public Sub LoadDictionary()
    '        Dim oTB As New TBAccess(oDB)
    '        Dim dt As DataTable

    '        dt = oTB.SqlDataTable("select * from dictionary")

    '        For ct As Integer = 0 To dt.Rows.Count - 1
    '            Me(dt.Rows(ct)("word")) = New DerivatelDictionaryElement(dt.Rows(ct)("wordId"), Nothing)
    '        Next


    '    End Sub

    '    Public l As New List(Of DerivatelDictionaryElement)

    'End Class
    'Public Class DerivatelDictionaryElement

    '    Public WordId As UInteger
    '    Public DerivateIds() As UInteger

    '    Public Sub New(ByVal id As Integer, ByVal DerIds() As Integer)

    '        Me.WordId = id

    '    End Sub



    'End Class

End Namespace
